package com.smartdo.crawler.processor;

import com.smartdo.crawler.utils.RedisFactory;
import org.apache.commons.lang3.StringUtils;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.scheduler.Scheduler;

/**
 * 资源调度
 *
 * @author zhangcheng
 */
public class SumaitongScheduler implements Scheduler {

    @Override
    public void push(Request request, Task task) {
        RedisFactory.setListValue("test:sumaitong:list", request.getUrl());
    }

    @Override
    public Request poll(Task task) {
        String url;
        while (true) {
            url = RedisFactory.getListValue("test:sumaitong:list");
            if (StringUtils.isBlank(url)) {
                break;
            }
            if (RedisFactory.sismember("test:sumaitong:detailsSet", url)) {
                continue;
            }
            break;
        }
        if (StringUtils.isNotBlank(url)) {
            return new Request(url);
        }
        return null;
    }
}
